ERDDAP Extraction

metadata: erddap_sst.yml

Published

2025-08-17 00:03 UTC

Code

# dir_extractr = "/share/github/marinebon/extractr"
# dir_extractr = "~/Github/marinebon/extractr" # DEBUG
# devtools::load_all(dir_extractr)
# devtools::install_local(dir_extractr, force = T)
# devtools::install_github("marinebon/extractr", force = T)
librarian::shelf(
  dplyr, DT, fs, glue, here, logger, lubridate, mapview, purrr, RColorBrewer, readr, 
  sf, stringr, terra, tibble, tidyr, units, yaml,
  marinebon/extractr,
  quiet = T)

options(readr.show_col_types = F)
mapviewOptions(
  basemaps       = "Esri.OceanBasemap",
  vector.palette = colorRampPalette(brewer.pal(n=5, name="Spectral")))

if (!exists("params"))
  params <- list(yml = here("meta/erddap_sst.yml")) # DEBUG

1 Polygons

Code

sanctuaries_rds <- here("../climate-dashboard/data/sanctuaries.rds")
buf_pct         <- 0.20 # 20% of area

ply_sanctuaries <- readRDS(sanctuaries_rds)

# buffer polygon and get bounding boxes
ply_sanctuaries <- ply_sanctuaries |>
  mutate(
    area_km2  = st_area(geom) |> 
      set_units("km^2") |> 
      as.numeric() |> 
      round(2),
    buf_km    = (sqrt(area_km2) * buf_pct) |> 
      round(2),
    bbox_geom = map2(geom, area_km2, \(g,a) {
      st_sfc(g, crs = 4326) |> 
        st_buffer(buf_km * 1000) |> 
        st_bbox() |>
        round(2) |> 
        st_as_sfc() }),
    bbox_chr  = map_chr(bbox_geom, \(x) {  # xmin, ymin, xmax and ymax
      st_bbox(x) |> 
        as.numeric() |> 
        paste(collapse = ",") } ) ) |> 
  unnest(bbox_geom)
bb_sanctuaries <- st_set_geometry(ply_sanctuaries, "bbox_geom") |> 
  select(-geom)
ply_sanctuaries <- select(ply_sanctuaries, -bbox_geom)

# show map
mapView(ply_sanctuaries, zcol = "nms") +
  mapView(bb_sanctuaries, zcol = "nms", legend = F)

Code

ply_sanctuaries |>
  st_drop_geometry() |>
  datatable()

2 Dataset

2.1 Metadata

Contents of erddap_sst.yml:

Code

cat(
  "```yaml",
  readLines(params$yml),
  "```",
  sep = "\n")

erddap_url: https://coastwatch.noaa.gov/erddap/griddap/noaacrwsstDaily.html
erddap_variable: analysed_sst

2.2 Information

Code

meta <- read_yaml(params$yml) 
proc_prod <- path_ext_remove(basename(params$yml))

dir_out <- glue("/share/data/noaa-onms/climate-dashboard-app/{proc_prod}")
dir_create(dir_out)

v     <- meta$erddap_variable
(ed   <- ed_info(meta$erddap_url))

<ERDDAP info> noaacrwsstDaily 
 Base URL: https://coastwatch.noaa.gov/erddap 
 Dataset Type: griddap 
 Dimensions (range):  
     time: (1985-01-01T12:00:00Z, 2025-08-14T12:00:00Z) 
     latitude: (-89.975, 89.975) 
     longitude: (-179.975, 179.975) 
 Variables:  
     analysed_sst: 
         Units: degree_C 
     sea_ice_fraction: 
         Units: 1

Code

times <- ed_dim(ed, "time")

3 Polygon years

Code

d_nms_yr_todo <- ply_sanctuaries |> 
  st_drop_geometry() |> 
  arrange(nms) |> 
  select(nms) |> 
  cross_join(
    tibble(
      year = year(min(times)):year(max(times)))) |> 
  mutate(
    d = map2(nms, year, \(nms, year) { # nms = "CBNMS"; year = 2025
      
      times_yr <- times[year(times) == year]
      tif      <- glue("{dir_out}/{nms}/{year}.tif")
      
      # if missing tif,return all times
      if (!file.exists(tif))
        return(list(
          time_min = min(times_yr),
          time_max = max(times_yr),
          n_times  = length(times_yr)))
      
      # get times from tif
      r <- rast(tif)
      times_tif <- time(r)
      
      # if all times done, return NAs
      if (all(times_yr %in% times_tif))
        return(list(
          n_times  = 0,
          time_min = NA,
          time_max= NA)) 
      
      # otherwise, return time range missing
      i <- !times_yr %in% times_tif
      list(
          n_times  = sum(i),
          time_min = min(times_yr[i]),
          time_max = max(times_yr[i])) 
      }),
    time_min = map_vec(d, pluck, "time_min"),
    time_max = map_vec(d, pluck, "time_max"),
    n_times  = map_int(d, pluck, "n_times") ) |> 
  select(-d) |> 
  filter(n_times > 0) |> 
  rowid_to_column("i") |>
  relocate(i)

d_nms_yr_todo |> 
  group_by(nms) %>%
  {if (nrow(.) > 0)
    summarize(
      .,
      time_min = min(time_min, na.rm = T),
      time_max = max(time_max, na.rm = T),
      n_times  = sum(n_times)) else .} |>
  datatable(
    caption  = "Sanctuaries missing available ERDDAP times.",
    rownames = F,
    options  = list(
      pageLength = 5,
      lengthMenu = c(5, 50, nrow(d_nms_yr_todo))))

4 Extract dataset per polygon year

Using extractr::ed_extract(), .

Code

# rerddap::cache_delete_all()

fxn <- function(i, nms, time_min, time_max, ...){ #  nms = "MNMS"; year = 2010 ; i = 97
  
  err <- tryCatch({
    
    log_info("{sprintf('%03d', i)}: {nms}, {time_min} to {time_max}")
    
    yr <- year(time_min)

    ply <- ply_sanctuaries |>
      filter(nms == !!nms)
    bb <- bb_sanctuaries |>
      filter(nms == !!nms) |> 
      st_bbox()
    
    extractr::ed_extract(
      ed,
      var       = v,
      sf_zones  = ply,
      bbox      = bb,
      mask_tif  = F,
      rast_tif  = glue("{dir_out}/{nms}/{yr}.tif"),
      zonal_fun = "mean",
      zonal_csv = glue("{dir_out}/{nms}/{yr}.csv"),
      dir_nc    = glue("{dir_out}/{nms}/{yr}_nc"),
      keep_nc   = F,
      n_max_vals_per_req = 1e+05,
      time_min  = time_min,
      time_max  = time_max,
      verbose   = T)
    
    return(NA)
  }, error = function(e) {
    
    log_error(conditionMessage(e))
    return(conditionMessage(e))
  })
  
  err
} 

res <- d_nms_yr_todo |> 
  mutate(
    error = pmap_chr(list(i, nms, time_min, time_max), fxn))

Downloading 1 requests, up to 111 time slices each

Fetching request 1 of 1 (2025-08-14 to 2025-08-14) ~ 00:01:25 UTC

Error in open.connection(structure(4L, class = c("curl", "connection"), conn_id = <pointer: 0x55a0670b5920>),  : 
  cannot open the connection
Error in open.connection(structure(5L, class = c("curl", "connection"), conn_id = <pointer: 0x55a068301540>),  : 
  cannot open the connection

Downloading 1 requests, up to 64 time slices each

Fetching request 1 of 1 (2025-08-11 to 2025-08-14) ~ 00:01:35 UTC

Downloading 1 requests, up to 31 time slices each

Fetching request 1 of 1 (2025-08-14 to 2025-08-14) ~ 00:01:42 UTC

Downloading 1 requests, up to 44 time slices each

Fetching request 1 of 1 (2025-08-14 to 2025-08-14) ~ 00:01:49 UTC

Downloading 1 requests, up to 392 time slices each

Fetching request 1 of 1 (2025-08-11 to 2025-08-14) ~ 00:01:55 UTC

Downloading 1 requests, up to 16 time slices each

Fetching request 1 of 1 (2025-08-11 to 2025-08-14) ~ 00:02:01 UTC

Error : Error {
    code=500;
    message="Internal Server Error: FileNotFoundException: /erddap/cache/ly/noaacrwsstDaily/8b06_afc1_ea0e_122225646_0 (No space left on device)";
}


RETRYing...

Downloading 1 requests, up to 27 time slices each

Fetching request 1 of 1 (2025-08-11 to 2025-08-14) ~ 00:02:11 UTC

Error : Error {
    code=500;
    message="Internal Server Error: FileNotFoundException: /erddap/cache/ly/noaacrwsstDaily/8ce1_e3a8_2a24_1227139517_0 (No space left on device)";
}


RETRYing...

Error : Error {
    code=500;
    message="Internal Server Error: FileNotFoundException: /erddap/cache/ly/noaacrwsstDaily/8ce1_e3a8_2a24_1726959750_0 (No space left on device)";
}


RETRYing...

Downloading 1 requests, up to 162 time slices each

Fetching request 1 of 1 (2025-08-11 to 2025-08-14) ~ 00:02:24 UTC

Downloading 1 requests, up to 28 time slices each

Fetching request 1 of 1 (2025-08-14 to 2025-08-14) ~ 00:02:30 UTC

Error : Error {
    code=503;
    message="Service Unavailable: There was a (temporary?) problem.  Wait a minute, then try again.  (In a browser, click the Reload button.)";
}


RETRYing...

Error in path_to_connection(x) : 
  'Error { code=503; message="Service Unavailable: There was a
(temporary?) problem.  Wait a minute, then try again.  (In a browser, click the
Reload button.)"; } ' does not exist in current working directory
('/share/github/noaa-onms/climate-dashboard-app/log').

ERROR in calling path_to_connection:
'Error {     code=503;     message="Service Unavailable: There was a (temporary?) problem.  Wait a minute, then try again.  (In a browser, click the Reload button.)"; } ' does not exist in current working directory ('/share/github/noaa-onms/climate-dashboard-app/log').ERROR in calling x:
'Error {     code=503;     message="Service Unavailable: There was a (temporary?) problem.  Wait a minute, then try again.  (In a browser, click the Reload button.)"; } ' does not exist in current working directory ('/share/github/noaa-onms/climate-dashboard-app/log').
RETRYing...

Error : Error {
    code=503;
    message="Service Unavailable: There was a (temporary?) problem.  Wait a minute, then try again.  (In a browser, click the Reload button.)";
}

Error in open.connection(structure(6L, class = c("curl", "connection"), conn_id = <pointer: 0x55a061674b00>),  : 
  cannot open the connection

Downloading 1 requests, up to 10 time slices each

Fetching request 1 of 1 (2025-08-14 to 2025-08-14) ~ 00:02:45 UTC

Downloading 1 requests, up to 47 time slices each

Fetching request 1 of 1 (2025-08-14 to 2025-08-14) ~ 00:02:51 UTC

Downloading 1 requests, up to 123 time slices each

Fetching request 1 of 1 (2025-08-14 to 2025-08-14) ~ 00:02:57 UTC

Error in open.connection(structure(7L, class = c("curl", "connection"), conn_id = <pointer: 0x55a0623f4b20>),  : 
  cannot open the connection

Warning: There were 4 warnings in `mutate()`.
The first warning was:
ℹ In argument: `error = pmap_chr(list(i, nms, time_min, time_max), fxn)`.
Caused by warning:
! Failed to open 'https://coastwatch.noaa.gov/erddap/griddap/noaacrwsstDaily.csvp?latitude': The requested URL returned error: 503
ℹ Run `dplyr::last_dplyr_warnings()` to see the 3 remaining warnings.

4.1 Successes

Code

res |> 
  mutate(
    success = is.na(error)) |> 
  group_by(success) |> 
  summarize(
    n = n()) |> 
  datatable()

4.2 Errors (if any)

Code

# rast(glue("{dir_out}/{nms}/{yr}.tif")) |> names()

res |> 
  filter(!is.na(error)) |> 
  group_by(nms) |> 
  summarize(
    n_years = n(),
    errors  = unique(error) |> paste(collapse = "\n\n----\n\n")) |> 
  datatable()

R package versions

Code

devtools::session_info()

─ Session info ───────────────────────────────────────────────────────────────
 setting  value
 version  R version 4.4.2 (2024-10-31)
 os       Ubuntu 24.04.1 LTS
 system   x86_64, linux-gnu
 ui       X11
 language (EN)
 collate  en_US.UTF-8
 ctype    en_US.UTF-8
 tz       Etc/UTC
 date     2025-08-17
 pandoc   3.5 @ /usr/bin/ (via rmarkdown)

─ Packages ───────────────────────────────────────────────────────────────────
 package           * version date (UTC) lib source
 base64enc           0.1-3   2015-07-28 [1] RSPM (R 4.4.0)
 bit                 4.6.0   2025-03-06 [1] RSPM (R 4.4.0)
 bit64               4.6.0-1 2025-01-16 [1] RSPM (R 4.4.0)
 brew                1.0-10  2023-12-16 [1] RSPM (R 4.4.0)
 bslib               0.9.0   2025-01-30 [1] RSPM (R 4.4.0)
 cachem              1.1.0   2024-05-16 [1] RSPM (R 4.4.0)
 class               7.3-22  2023-05-03 [2] CRAN (R 4.4.2)
 classInt            0.4-11  2025-01-08 [1] RSPM (R 4.4.0)
 cli                 3.6.5   2025-04-23 [1] RSPM (R 4.4.0)
 codetools           0.2-20  2024-03-31 [2] CRAN (R 4.4.2)
 colorspace          2.1-1   2024-07-26 [1] RSPM (R 4.4.0)
 crayon              1.5.3   2024-06-20 [1] RSPM (R 4.4.0)
 crosstalk           1.2.1   2023-11-23 [1] RSPM (R 4.4.0)
 crul                1.5.0   2024-07-19 [1] RSPM (R 4.4.0)
 curl                6.2.2   2025-03-24 [1] RSPM (R 4.4.0)
 data.table          1.17.2  2025-05-12 [1] RSPM (R 4.4.0)
 DBI                 1.2.3   2024-06-02 [1] RSPM (R 4.4.0)
 deldir              2.0-4   2024-02-28 [1] RSPM (R 4.4.0)
 devtools            2.4.5   2022-10-11 [1] RSPM (R 4.4.0)
 dichromat           2.0-0.1 2022-05-02 [1] RSPM (R 4.4.0)
 digest              0.6.37  2024-08-19 [1] RSPM (R 4.4.0)
 dplyr             * 1.1.4   2023-11-17 [1] RSPM (R 4.4.0)
 DT                * 0.33    2024-04-04 [1] RSPM (R 4.4.0)
 dygraphs            1.1.1.6 2018-07-11 [1] RSPM (R 4.4.0)
 e1071               1.7-16  2024-09-16 [1] RSPM (R 4.4.0)
 ellipsis            0.3.2   2021-04-29 [1] RSPM (R 4.4.0)
 evaluate            1.0.3   2025-01-10 [1] RSPM (R 4.4.0)
 extractr          * 0.1.7   2025-05-19 [1] local (/share/github/marinebon/extractr)
 farver              2.1.2   2024-05-13 [1] RSPM (R 4.4.0)
 fastmap             1.2.0   2024-05-15 [1] RSPM (R 4.4.0)
 fs                * 1.6.6   2025-04-12 [1] RSPM (R 4.4.0)
 generics            0.1.4   2025-05-09 [1] RSPM (R 4.4.0)
 glue              * 1.8.0   2024-09-30 [1] RSPM (R 4.4.0)
 here              * 1.0.1   2020-12-13 [1] RSPM (R 4.4.0)
 hms                 1.1.3   2023-03-21 [1] RSPM (R 4.4.0)
 hoardr              0.5.5   2025-01-18 [1] RSPM (R 4.4.0)
 htmltools           0.5.8.1 2024-04-04 [1] RSPM (R 4.4.0)
 htmlwidgets         1.6.4   2023-12-06 [1] RSPM (R 4.4.0)
 httpcode            0.3.0   2020-04-10 [1] RSPM (R 4.4.0)
 httpuv              1.6.16  2025-04-16 [1] RSPM (R 4.4.0)
 jquerylib           0.1.4   2021-04-26 [1] RSPM (R 4.4.0)
 jsonlite            2.0.0   2025-03-27 [1] RSPM (R 4.4.0)
 KernSmooth          2.23-24 2024-05-17 [2] CRAN (R 4.4.2)
 knitr               1.50    2025-03-16 [1] RSPM (R 4.4.0)
 later               1.4.2   2025-04-08 [1] RSPM (R 4.4.0)
 lattice             0.22-6  2024-03-20 [2] CRAN (R 4.4.2)
 leafem              0.2.4   2025-05-01 [1] RSPM (R 4.4.0)
 leaflet             2.2.2   2024-03-26 [1] RSPM (R 4.4.0)
 leaflet.providers   2.0.0   2023-10-17 [1] RSPM (R 4.4.0)
 leafpop             0.1.0   2021-05-22 [1] RSPM (R 4.4.0)
 librarian           1.8.1   2021-07-12 [1] RSPM (R 4.4.0)
 lifecycle           1.0.4   2023-11-07 [1] RSPM (R 4.4.0)
 logger            * 0.4.0   2024-10-22 [1] RSPM (R 4.4.0)
 lubridate         * 1.9.4   2024-12-08 [1] RSPM (R 4.4.0)
 magrittr            2.0.3   2022-03-30 [1] RSPM (R 4.4.0)
 mapview           * 2.11.2  2023-10-13 [1] RSPM (R 4.4.0)
 Matrix              1.7-1   2024-10-18 [2] CRAN (R 4.4.2)
 memoise             2.0.1   2021-11-26 [1] RSPM (R 4.4.0)
 mime                0.13    2025-03-17 [1] RSPM (R 4.4.0)
 miniUI              0.1.1.1 2018-05-18 [1] RSPM (R 4.4.0)
 ncdf4               1.24    2025-03-25 [1] RSPM (R 4.4.0)
 pillar              1.10.2  2025-04-05 [1] RSPM (R 4.4.0)
 pkgbuild            1.4.5   2024-10-28 [1] RSPM (R 4.4.0)
 pkgconfig           2.0.3   2019-09-22 [1] RSPM (R 4.4.0)
 pkgload             1.4.0   2024-06-28 [1] RSPM (R 4.4.0)
 png                 0.1-8   2022-11-29 [1] RSPM (R 4.4.0)
 polyclip            1.10-7  2024-07-23 [1] RSPM (R 4.4.0)
 profvis             0.4.0   2024-09-20 [1] RSPM (R 4.4.0)
 promises            1.3.2   2024-11-28 [1] RSPM (R 4.4.0)
 proxy               0.4-27  2022-06-09 [1] RSPM (R 4.4.0)
 purrr             * 1.0.4   2025-02-05 [1] RSPM (R 4.4.0)
 R.methodsS3         1.8.2   2022-06-13 [1] RSPM (R 4.4.0)
 R.oo                1.27.1  2025-05-02 [1] RSPM (R 4.4.0)
 R.utils             2.13.0  2025-02-24 [1] RSPM (R 4.4.0)
 R6                  2.6.1   2025-02-15 [1] RSPM (R 4.4.0)
 rappdirs            0.3.3   2021-01-31 [1] RSPM (R 4.4.0)
 raster              3.6-32  2025-03-28 [1] RSPM (R 4.4.0)
 RColorBrewer      * 1.1-3   2022-04-03 [1] RSPM (R 4.4.0)
 Rcpp                1.0.14  2025-01-12 [1] RSPM (R 4.4.0)
 readr             * 2.1.5   2024-01-10 [1] RSPM (R 4.4.0)
 remotes             2.5.0   2024-03-17 [1] RSPM (R 4.4.0)
 rerddap             1.2.1   2025-03-19 [1] RSPM (R 4.4.0)
 rlang               1.1.6   2025-04-11 [1] RSPM (R 4.4.0)
 rmarkdown           2.29    2024-11-04 [1] RSPM (R 4.4.0)
 rprojroot           2.0.4   2023-11-05 [1] RSPM (R 4.4.0)
 s2                  1.1.9   2025-05-23 [1] RSPM (R 4.4.0)
 sass                0.4.10  2025-04-11 [1] RSPM (R 4.4.0)
 satellite           1.0.5   2024-02-10 [1] RSPM (R 4.4.0)
 scales              1.4.0   2025-04-24 [1] RSPM (R 4.4.0)
 sessioninfo         1.2.2   2021-12-06 [1] RSPM (R 4.4.0)
 sf                * 1.0-21  2025-05-15 [1] RSPM (R 4.4.2)
 shiny               1.10.0  2024-12-14 [1] RSPM (R 4.4.0)
 sp                  2.2-0   2025-02-01 [1] RSPM (R 4.4.0)
 spatstat.data       3.1-6   2025-03-17 [1] RSPM (R 4.4.0)
 spatstat.geom       3.3-6   2025-03-18 [1] RSPM (R 4.4.0)
 spatstat.univar     3.1-3   2025-05-08 [1] RSPM (R 4.4.0)
 spatstat.utils      3.1-4   2025-05-15 [1] RSPM (R 4.4.0)
 stringi             1.8.7   2025-03-27 [1] RSPM (R 4.4.0)
 stringr           * 1.5.1   2023-11-14 [1] RSPM (R 4.4.0)
 svglite             2.2.1   2025-05-12 [1] RSPM (R 4.4.0)
 systemfonts         1.2.3   2025-04-30 [1] RSPM (R 4.4.0)
 tabularaster        0.7.2   2023-11-01 [1] RSPM (R 4.4.0)
 terra             * 1.8-54  2025-06-01 [1] RSPM (R 4.4.0)
 textshaping         1.0.1   2025-05-01 [1] RSPM (R 4.4.0)
 tibble            * 3.3.0   2025-06-08 [1] RSPM (R 4.4.0)
 tidyr             * 1.3.1   2024-01-24 [1] RSPM (R 4.4.0)
 tidyselect          1.2.1   2024-03-11 [1] RSPM (R 4.4.0)
 timechange          0.3.0   2024-01-18 [1] RSPM (R 4.4.0)
 triebeard           0.4.1   2023-03-04 [1] RSPM (R 4.4.0)
 tzdb                0.5.0   2025-03-15 [1] RSPM (R 4.4.0)
 units             * 0.8-7   2025-03-11 [1] RSPM (R 4.4.0)
 urlchecker          1.0.1   2021-11-30 [1] RSPM (R 4.4.0)
 urltools            1.7.3   2019-04-14 [1] RSPM (R 4.4.0)
 usethis             3.1.0   2024-11-26 [1] RSPM (R 4.4.0)
 uuid                1.2-1   2024-07-29 [1] RSPM (R 4.4.0)
 vctrs               0.6.5   2023-12-01 [1] RSPM (R 4.4.0)
 vroom               1.6.5   2023-12-05 [1] RSPM (R 4.4.0)
 withr               3.0.2   2024-10-28 [1] RSPM (R 4.4.0)
 wk                  0.9.4   2024-10-11 [1] RSPM (R 4.4.0)
 xfun                0.52    2025-04-02 [1] RSPM (R 4.4.0)
 xml2                1.3.8   2025-03-14 [1] RSPM (R 4.4.0)
 xtable              1.8-4   2019-04-21 [1] RSPM (R 4.4.0)
 xts                 0.14.1  2024-10-15 [1] RSPM (R 4.4.0)
 yaml              * 2.3.10  2024-07-26 [1] RSPM (R 4.4.0)
 zoo                 1.8-14  2025-04-10 [1] RSPM (R 4.4.0)

 [1] /usr/local/lib/R/site-library
 [2] /usr/local/lib/R/library

──────────────────────────────────────────────────────────────────────────────